/************************************************************************
*************************************************************************
*************************************************************************

structural_idtests.do

The additional identification tests I concocted, all collected in one place.


20170320: 	Created (@)
20170326: 	Added the two-step estimator (@)
20170708: 	Made a version of the share test that 
			corrects for measurement error (@)

*************************************************************************
*************************************************************************
************************************************************************/

do "do/programs/ranktest_hacked.ado"


/************************************************************************
Program: widstat_iv
Purpose: Estimates rho (consistently), then uses it to construct the
			Kleibergen-Paap statistic
************************************************************************/


cap program drop widstat_iv_old
program widstat_iv_old
		preserve
		sort id t
		ar_dem
		local rhoest = _b[/rho]
		foreach inst of varlist k-mXm {
			gen ENDO2_`inst' = `inst' - `rhoest'*l.`inst'
			gen INST_`inst' = l.`inst'
		}
		
		gen yrho2 = y - `rhoest'*l.y
		
		ivreg2 y (ENDO2_* = INST_* l2.m l2.k l2.kXm), cluster(id)
		restore
		gen widstat_iv_old = e(widstat)
	
end


cap program drop widstat_iv
program widstat_iv
		preserve
		sort id t
		ar_dem
		local rhoest = _b[/rho]
		foreach inst of varlist k-mXm {
			gen ENDO2_`inst' = `inst' - `rhoest'*l.`inst'
			gen INST_`inst' = l.`inst'
		}
		
		gen yrho2 = y - `rhoest'*l.y
		
		ivreg2 yrho2 (ENDO2_* = INST_* l2.m l2.k l2.kXm), cluster(id)
		local wid = e(widstat)
		local dofadj = (`e(N)'-`e(inexog_ct)'-`e(exexog_ct)')/(`e(N)'-1) * ( `e(N_clust)'-1)/`e(N_clust)' / `e(exexog_ct)'
		
		ranktest_hacked (ENDO2_*) (INST_* l2.m l2.k l2.kXm), cluster(id) wald fullrank
		cap matrix drop labest
		matrix labest = r(lab)'
		
		restore
		gen widstat_iv = `wid'
		gen dofadj = `dofadj'
		svmat labest
	
end



/************************************************************************
Program: widstat_iv_instlist
Purpose: Estimates rho (consistently), then uses it to construct the
			Kleibergen-Paap statistic. This version uses a custom number
			of instruments.
************************************************************************/

cap program drop widstat_iv_instlist
program widstat_iv_instlist
	args instnum
	
	
	local ilist
	local count 0
	foreach term in l2.m l2.k l2.kXm l2.l l2.lXm l2.kXl {
		local ++count
		
		local ilist `ilist' `term'
		local dlaglist`count' `ilist'
	}
	
	
	
	preserve
	sort id t
	ar_dem_modinst "`dlaglist`instnum''"
	local rhoest = _b[/rho]
	foreach inst of varlist k-mXm {
		gen ENDO2_`inst' = `inst' - `rhoest'*l.`inst'
		gen INST_`inst' = l.`inst'
	}
	
	gen yrho2 = y - `rhoest'*l.y
	
	ivreg2 yrho2 (ENDO2_* = INST_* `dlaglist`instnum''), cluster(id)
	restore
	gen widstat_iv`instnum' = e(widstat)

end

cap program drop widstat_iv_instlist_old
program widstat_iv_instlist_old
	args instnum
	
	
	local ilist
	local count 0
	foreach term in l2.m l2.k l2.kXm l2.l l2.lXm l2.kXl {
		local ++count
		
		local ilist `ilist' `term'
		local dlaglist`count' `ilist'
	}
	
	
	
	preserve
	sort id t
	ar_dem_modinst "`dlaglist`instnum''"
	local rhoest = _b[/rho]
	foreach inst of varlist k-mXm {
		gen ENDO2_`inst' = `inst' - `rhoest'*l.`inst'
		gen INST_`inst' = l.`inst'
	}
	
	gen yrho2 = y - `rhoest'*l.y
	
	ivreg2 y (ENDO2_* = INST_* `dlaglist`instnum''), cluster(id)
	restore
	gen widstat_iv_old`instnum' = e(widstat)

end




/************************************************************************
Program: jacform
Purpose: Forms the jacobian of the moment conditions
			Must be run after ar_dem
************************************************************************/


cap program drop jacform
program jacform

	local deriv1  	 -k 	+ _b[/rho]*l.k 
	local deriv2  	 -l 	+ _b[/rho]*l.l
	local deriv3  	 -m 	+ _b[/rho]*l.m
	local deriv4  	 -kXk 	+ _b[/rho]*l.kXk
	local deriv5  	 -lXl 	+ _b[/rho]*l.lXl
	local deriv6  	 -mXm 	+ _b[/rho]*l.mXm
	local deriv7  	 -kXl 	+ _b[/rho]*l.kXl
	local deriv8  	 -kXm 	+ _b[/rho]*l.kXm
	local deriv9  	 -lXm 	+ _b[/rho]*l.lXm
	local deriv10  	-(l.y - _b[/bk]*l.k - _b[/bl]*l.l - _b[/bm]*l.m - _b[/bkk]*l.kXk - _b[/bll]*l.lXl - _b[/bmm]*l.mXm - _b[/bkl]*l.kXl - _b[/bkm]*l.kXm - _b[/blm]*l.lXm)
	
	tempname fakeid
	gen `fakeid' = 1
	
	tempfile pre
	save `pre'
	
	tempvar tmp
	foreach var of varlist y k l m kXk lXl mXm kXl kXm lXm {
		egen `tmp' = mean(`var')
		replace `var' = `var'-`tmp'
		drop `tmp'
	}
	
	xtset
	local pv = r(panelvar)
	local tv = r(timevar)
	xtset `pv' `tv'
	foreach d of numlist 1/10 {
		local count 0
		foreach inst in L.k L.l L.m L.kXk L.kXl L.kXm L.lXl L.lXm L.mXm L2.m L2.k L2.kXm 1 {
			local ++count
			gen r`count'c`d' = (`deriv`d'')*`inst'
		}

	}
	


	collapse r*c*
	
	rename _all jacmat_=
	
	gen `fakeid' = 1
	
	tempfile jacmat
	save `jacmat'
	
	use `pre', clear
	
	//The observation is non-missing only for the first obs
	merge m:1 `fakeid' using `jacmat'
	drop `fakeid' _merge
	
end



/************************************************************************
Program: singular_diagnose
Purpose: Gets diagnostics to help determine whether the moment conditions
			are flat with respect to some key value. 
			Output produced: singular values, R-squared of regression of
			pm derivative on pk and pl derivatives
************************************************************************/


cap program drop singular_diagnose
program singular_diagnose

	local deriv1  	 -k 	+ _b[/rho]*l.k 
	local deriv2  	 -l 	+ _b[/rho]*l.l
	local deriv3  	 -m 	+ _b[/rho]*l.m
	local deriv4  	 -kXk 	+ _b[/rho]*l.kXk
	local deriv5  	 -lXl 	+ _b[/rho]*l.lXl
	local deriv6  	 -mXm 	+ _b[/rho]*l.mXm
	local deriv7  	 -kXl 	+ _b[/rho]*l.kXl
	local deriv8  	 -kXm 	+ _b[/rho]*l.kXm
	local deriv9  	 -lXm 	+ _b[/rho]*l.lXm
	local deriv10  	-(l.y - _b[/bk]*l.k - _b[/bl]*l.l - _b[/bm]*l.m - _b[/bkk]*l.kXk - _b[/bll]*l.lXl - _b[/bmm]*l.mXm - _b[/bkl]*l.kXl - _b[/bkm]*l.kXm - _b[/blm]*l.lXm)

	preserve
	
	tempvar tmp
	foreach var of varlist y k l m kXk lXl mXm kXl kXm lXm {
		egen `tmp' = mean(`var')
		replace `var' = `var'-`tmp'
		drop `tmp'
	}
	
	sort id t
	foreach d of numlist 1/10 {
		local count 0
		foreach inst in L.k L.l L.m L.kXk L.kXl L.kXm L.lXl L.lXm L.mXm L2.m L2.k L2.kXm 1 {
			local ++count
			gen r`count'c`d' = (`deriv`d'')*`inst'
		}

	}
	
	
	forvalues i=1/13 {
		reg r`i'c3 r`i'c1 r`i'c2
		local vec`i'r2 = e(r2)
		
		reg r`i'c3 r`i'c1 r`i'c2 r`i'c4-r`i'c10
		local vecfull`i'r2 = e(r2)
	}
	
	tempfile precollapse
	save `precollapse'
	

	collapse r*c*

	gen i = 1
	reshape long @c1 @c2 @c3 @c4 @c5 @c6 @c7 @c8 @c9 @c10, i(i) j(row) string


	mkmat c*, matrix(jacmom)
	
	matrix svd U W V = jacmom
	
	
	
	//Stacked regression
	use `precollapse', clear
	keep id t r*c*
	reshape long r@c1 r@c2 r@c3 r@c4 r@c5 r@c6 r@c7 r@c8 r@c9 r@c10, i(id t) j(row)
	
	reg rc3 rc1 rc2 rc4- rc10
	local jacstackr2 = e(r2)

	
	restore
	
	svmat W, names(singval)
	
	forvalues i=1/13 {
		gen jacreg`i' 		= `vec`i'r2'
		gen jacfullreg`i' 	= `vecfull`i'r2'
	}
	
	gen jacstack			= `jacstackr2'
end




/************************************************************************
Program: direct_id_test
Purpose: Directly tests whether the derivative condition for the coefficient
			on m is a linear combination of the other conditions.
************************************************************************/


cap program drop direct_id_test
program direct_id_test
	preserve
	
		/************
		Construct Variables
		************/

		sort id t
		gen lagk = l.k
		gen lagm = l.m
		gen lagl = l.l
		gen lagm2 = l2.m
		
		
		//This part of the statement will be used in the full model
		local statement2
		local testlist
		local count 0
		foreach var in L.k L.l L.m L.kXk L.kXl L.kXm L.lXl L.lXm L.mXm L2.m L2.k L2.kXm 1 {
			local ++count
			
			gen INST`count' = `var'
			local statement2 "`statement2' + {i`count'}*INST`count'"
			
			local testlist "`testlist' [i`count']_cons"
		}
		
		
		
		/************
		De-Mean Sample
		************/
		
		tempvar tmp
		foreach var of varlist m lagm lagk lagl INST* {
			egen `tmp' = mean(`var')
			replace `var' = `var'-`tmp'
			drop `tmp'
		}
		
		
		/************
		Model Test 1: Only l2.m as an additional instrument
		************/
		
		//Restrict sample
		drop if mi(lagk,lagm, lagl, lagm2)
		
		//nl (m = {cons} + {rho}*lagm + {a}*({rho}*lagk - k) + {b}*({rho}*lagl - l) )
		nl (m =  {rho}*lagm + {a}*({rho}*lagk - k) + {b}*({rho}*lagl - l) )
		local red_r2 = e(r2)
		local red_rss = e(rss)
		
		drop if mi(lagm2)
		//nl (m = {cons} + {rho}*lagm + {a}*({rho}*lagk - k) + {b}*({rho}*lagl - l) + {c}*lagm2 )
		nl (m = {rho}*lagm + {a}*({rho}*lagk - k) + {b}*({rho}*lagl - l) + {c}*lagm2 )
		local ful1_r2 = e(r2)
		local ful1_rss = e(rss)
		test [c]_cons
		local test1 = r(F)
		local pval1 = r(p)
		
		
		/************
		Model Test 2: Full set of instruments
		************/
		
		//Restrict sample
		egen anymiss = rowmiss(m INST*)
		drop if anymiss > 0
		

		//nl (m = {cons} + {rho}*lagm + {a}*({rho}*lagk - k) + {b}*({rho}*lagl - l) )
		nl (m = {rho}*lagm + {a}*({rho}*lagk - k) + {b}*({rho}*lagl - l) )
		local red_r2_2 = e(r2)
		local red_rss_2 = e(rss)
		
		//nl (m = {cons} + {rho}*lagm + {a}*({rho}*lagk - k) + {b}*({rho}*lagl - l) `statement2' 
		nl (m = {rho}*lagm + {a}*({rho}*lagk - k) + {b}*({rho}*lagl - l) `statement2' )
		local ful2_r2 = e(r2)
		local ful2_rss = e(rss)
		
		test `testlist'
		local teststat 	= r(F)
		local testp		= r(p)
		
	restore
	
	gen idtest1_red_r2 	= `red_r2'
	gen idtest1_ful_r2 	= `ful1_r2'
	
	gen idtest2_red_r2	= `red_r2_2'
	gen idtest2_ful_r2 	= `ful2_r2'
	
	gen idtest1_pr2 	= (`red_rss'-`ful1_rss')/`red_rss'
	gen idtest2_pr2 	= (`red_rss_2'-`ful2_rss')/`red_rss'
	
	gen idtest1_F		= `test1'
	gen idtest1_p		= `pval1'
	
	gen idtest2_F		= `teststat'
	gen idtest2_p		= `testp'
end



/************************************************************************
Program: direct_id_test_adv
Purpose: Runs a more sophisticated test of direct identification
************************************************************************/

cap program drop direct_id_test_adv
program direct_id_test_adv
	syntax varlist [if], Instruments(varlist) [stub(name)]
	
	//If they don't pass a stub, fill something in
	if mi("`stub'") {
		local stub idtest_adv
	}
	
	//Get intermediates and all the other variables (terms in production function)
	gettoken m rest: varlist
	
	preserve

		sort id t
		
		//Construct lags and also build up a statement
		local statement ""
		local count 0
		foreach var of varlist `rest' {
			gen LAG`var' = l.`var'
			
			//Build up NL statement
			local ++count
			local statement "`statement' + {a`count'}*( {rho}*LAG`var' - `var'   )"
		}
		
		gen LAG`m' = l.`m'
		
		
		//This part of the statement will be used in the full model
		local statement2
		local testlist
		local count 0
		foreach var of varlist `instruments' {
			local ++count
			local statement2 "`statement2' + {b`count'}*`var'"
			
			local testlist "`testlist' [b`count']_cons"
		}
		
		
		//Drop anything with missing variables
		egen anymiss = rowmiss(`m' `rest' LAG* `instruments')
		drop if anymiss > 0
		
		
		
		//Demean
		tempvar tmp
		foreach var of varlist `m' `rest' LAG* `instruments' {
			egen `tmp' = mean(`var')
			replace `var' = `var'-`tmp'
			drop `tmp'
		}
		
		
		//Reduced model
		//nl (`m' = {cons} + {rho}*LAG`m' `statement' ), cluster(id)
		nl (`m' =  {rho}*LAG`m' `statement' ), cluster(id)
		local red_r2 	= e(r2)
		local red_rss	= e(rss)
		
		//Full model
		//nl (`m' = {cons} + {rho}*LAG`m' `statement' `statement2'), cluster(id)
		nl (`m' =  {rho}*LAG`m' `statement' `statement2'), cluster(id)
		local ful_r2	= e(r2)
		local full_rss = e(rss)
		
		test `testlist'
		local teststat 	= r(F)
		local testp		= r(p)
		
	restore
	
	gen `stub'_red_r2 	= `red_r2'
	gen `stub'_ful_r2 	= `ful_r2'
	gen `stub'_pr2	= (`red_rss'-`full_rss')/`red_rss'
	gen `stub'_F	= `teststat'
	gen `stub'_pval	= `testp'
end




/************************************************************************
Program: direct_id_test_inst
Purpose: Like direct_id_test_adv, but uses GMM and instruments the difference
			terms with only the lags.
************************************************************************/

cap program drop direct_id_test_inst
program direct_id_test_inst
	syntax varlist [if], Instruments(varlist) [stub(name)]
	
	//If they don't pass a stub, fill something in
	if mi("`stub'") {
		local stub idtest_inst
	}
	
	//Get intermediates and all the other variables (terms in production function)
	gettoken m rest: varlist
	
	preserve

		sort id t
		
		gen LAG`m' = l.`m'
		
		//Construct lags and also build up a statement, a list of derivative conditions, and the list of instruments
		local statement 	"{rho}*LAG`m'"
		local derivcoef 	"" 			//The derivatives for a1-aN
		local derivrho		"/rho = -LAG`m'"	//The derivative for rho
		local instlist		"LAG`m'"
		local count 0
		foreach var of varlist `rest' {
			gen LAG`var' = l.`var'
			
			//Build up NL statement
			local ++count
			
			/*if `count' == 1 {
				local statement "`statement' {a`count'}*( `var'  - {rho}*LAG`var' )"
			}
			else {
				local statement "`statement' + {a`count'}*( `var'  - {rho}*LAG`var' )"
			}*/
			
			local statement "`statement' + {a`count'}*( `var'  - {rho}*LAG`var' )"
			
			local derivcoef "`derivcoef' deriv(/a`count' = {rho}*LAG`var' - `var')"
			local derivrho	"`derivrho' + {a`count'}*LAG`var'"
			
			local instlist		"`instlist' LAG`var'"
		}
		
		
		
		
		//This part of the statement will be used in the full model
		local statement2
		local derivcoef2
		local testlist
		local count 0
		foreach var of varlist `instruments' {
			local ++count
			local statement2 "`statement2' + {b`count'}*`var'"
			
			local testlist "`testlist' [b`count']_cons"
			
			local derivcoef2 "`derivcoef2' deriv(/b`count' = -`var')"
		}
		
		
		//Drop anything with missing variables
		egen anymiss = rowmiss(`m' `rest' LAG* `instruments')
		drop if anymiss > 0
		
		
		
		//Demean
		tempvar tmp
		foreach var of varlist `m' `rest' LAG* `instruments' {
			egen `tmp' = mean(`var')
			replace `var' = `var'-`tmp'
			drop `tmp'
		}
		
		
		//Reduced model
		//nl (`m' = {cons} + {rho}*LAG`m' `statement' ), cluster(id)
		//nl (`m' =  {rho}*LAG`m' `statement' ), cluster(id)
		/*noisily: di "gmm (`m' - (`statement') )"
		noisily: di "--------------------------------------------------------"
		noisily: di "instruments(`instlist')"
		noisily: di "--------------------------------------------------------"
		noisily: di "`derivcoef' deriv(`derivrho')"
		 */
		
		cap gmm (`m' - (`statement') ), instruments(`instlist')  ///
				`derivcoef' deriv(`derivrho')
		predict RES1 if e(sample)
		gen SQRES1	= RES1^2
		
		egen RSS1 = total(SQRES1)
		sum RSS1
		local red_rss	= r(mean)
		
		egen MEANm 	= mean(`m')				if e(sample)
		gen SQUAREm = (`m'-MEANm)^2			if e(sample)
		egen TOTSS 	= total(SQUAREm)
		sum TOTSS
		local var		= r(mean)
		local red_r2 	= 1 - `red_rss'/`var'
		
		//Full model
		//nl (`m' = {cons} + {rho}*LAG`m' `statement' `statement2'), cluster(id)
		//nl (`m' =  {rho}*LAG`m' `statement' `statement2'), cluster(id)
		cap gmm (`m' - (`statement' `statement2') ), instruments(`instlist' `instruments')  ///
				`derivcoef' `derivcoef2' deriv(`derivrho')
		
		predict RES2 if e(sample)
		gen SQRES2		= RES2^2
		
		egen RSS2 		= total(SQRES2)
		sum RSS2
		local full_rss 	= r(mean)
		local ful_r2 	= 1 - `full_rss'/`var'
		
		test `testlist'
		local teststat 	= r(chi2)
		local testp		= r(p)
		
	restore
	
	gen `stub'_red_r2 	= `red_r2'
	gen `stub'_ful_r2 	= `ful_r2'
	gen `stub'_pr2	= (`red_rss'-`full_rss')/`red_rss'
	gen `stub'_F	= `teststat'
	gen `stub'_pval	= `testp'
end


/************************************************************************
Program: direct_id_test_simple
Purpose: Similar derivation to the other direct ID tests, but this one
			just uses OLS.
************************************************************************/

cap program drop direct_id_test_simple
program direct_id_test_simple
	syntax varlist [if], Instruments(varlist) [stub(name)]
	
	//If they don't pass a stub, fill something in
	if mi("`stub'") {
		local stub idtest_simple
	}
	
	//Get intermediates and all the other variables (terms in production function)
	gettoken m rest: varlist
	
	preserve

		sort id t
		
		gen LAG`m' = l.`m'
		
		//Construct lags and also build up a statement, a list of derivative conditions, and the list of instruments
		local laglist LAG`m'
		foreach var of varlist `rest' {
			gen LAG`var' = l.`var'
			local laglist `laglist' LAG`var'
		}
		
		
		
		//Drop anything with missing variables
		egen anymiss = rowmiss(`m' `rest' LAG* `instruments')
		drop if anymiss > 0
		
		
		
		//Demean
		tempvar tmp
		foreach var of varlist `m' `rest' LAG* `instruments' {
			egen `tmp' = mean(`var')
			replace `var' = `var'-`tmp'
			drop `tmp'
		}
		
		
		//Reduced model
		
		reg `m' `rest' `laglist', cluster(id)
		local red_r2 	= e(r2)
		local red_rss	= e(rss)
		
		//Full model
		reg `m' `rest' `laglist' `instruments', cluster(id)
		local ful_r2	= e(r2)
		local full_rss = e(rss)
		
		testparm `instruments'
		local teststat 	= r(F)
		local testp		= r(p)
		
	restore
	
	gen `stub'_red_r2 	= `red_r2'
	gen `stub'_ful_r2 	= `ful_r2'
	gen `stub'_pr2	= (`red_rss'-`full_rss')/`red_rss'
	gen `stub'_F	= `teststat'
	gen `stub'_pval	= `testp'
end




/************************************************************************
Program: direct_id_test_share
Purpose: Based on the share regression, which precludes the need for
			controlling for lags.
************************************************************************/

cap program drop direct_id_test_share
program direct_id_test_share
	syntax varlist [if], Instruments(varlist) [stub(name)]
	
	//If they don't pass a stub, fill something in
	if mi("`stub'") {
		local stub idtest_share
	}
	
	//Get intermediates and all the other variables (terms in production function)
	gettoken s rest: varlist
	
	preserve

		sort id t
		
		
		
				
		
		
		//Drop anything with missing variables
		egen anymiss = rowmiss(`s' `rest' `instruments')
		drop if anymiss > 0
		
		
		
		
		//Demean
		tempvar tmp
		foreach var of varlist `s' `rest' `instruments' {
			egen `tmp' = mean(`var')
			replace `var' = `var'-`tmp'
			drop `tmp'
		}
		
		
		//Reduced model
		
		reg `s' `rest' , cluster(id)
		local red_r2 	= e(r2)
		local red_rss	= e(rss)
		
		//Full model
		reg `s' `rest' `instruments', cluster(id)
		local ful_r2	= e(r2)
		local full_rss = e(rss)
		
		testparm `instruments'
		local teststat 	= r(F)
		local testp		= r(p)
		
	restore
	
	gen `stub'_red_r2 	= `red_r2'
	gen `stub'_ful_r2 	= `ful_r2'
	gen `stub'_pr2	= (`red_rss'-`full_rss')/`red_rss'
	gen `stub'_F	= `teststat'
	gen `stub'_pval	= `testp'
end





/************************************************************************
Program: direct_id_test_share_mmerr
Purpose: Based on the share regression, which precludes the need for
			controlling for lags. But this version deals with measurement
			error by instrumenting the current terms with their lags.
************************************************************************/

cap program drop direct_id_test_share_mmerr
program direct_id_test_share_mmerr
	syntax varlist [if], INSTruments(varlist) TESTvars(varlist) [ITESTvars(varlist) stub(name)]
	
	//If they don't pass a stub, fill something in
	if mi("`stub'") {
		local stub idtest_smmerr
	}
	
	//Get intermediates and all the other variables (terms in production function)
	gettoken s rest: varlist
	
	preserve

		sort id t
		
		
		
				
		
		
		//Drop anything with missing variables
		egen anymiss = rowmiss(`s' `rest' `instruments' `testvars' `itestvars')
		drop if anymiss > 0
		
		
		
		
		//Demean
		tempvar tmp
		foreach var of varlist `s' `rest' `instruments' `testvars' `itestvars' {
			egen `tmp' = mean(`var')
			replace `var' = `var'-`tmp'
			drop `tmp'
		}
		
		
		//Reduced model
		
		ivreg2 `s' (`rest'=`instruments') , cluster(id) small
		local red_r2 	= e(r2)
		local red_rss	= e(rss)
		
		//Full model
		if "`itestvars'" == "" {
			ivreg2 `s' `testvars' (`rest'=`instruments') , cluster(id) small
		}
		else {
			ivreg2 `s' (`rest' `testvars'=`instruments' `itestvars') , cluster(id) small
		}
		
		local ful_r2	= e(r2)
		local full_rss = e(rss)
		
		testparm `testvars'
		local teststat 	= r(F)
		local testp		= r(p)
		
	restore
	
	gen `stub'_red_r2 	= `red_r2'
	gen `stub'_ful_r2 	= `ful_r2'
	gen `stub'_pr2	= (`red_rss'-`full_rss')/`red_rss'
	gen `stub'_F	= `teststat'
	gen `stub'_pval	= `testp'
end


/************************************************************************
Program: hybrid_test
Purpose: Runs an ID test that combines the idea behind the jacobian test
			and the direct ID test
			

************************************************************************/


cap program drop hybrid_test
program hybrid_test
	syntax [if], Instruments(varlist fv ts) [stub(name)]
	
	
	if mi("`stub'") {
		local stub hybrid_test
	}

	local deriv1  	 -m 	+ ar_gnr*l.m
	local deriv2  	 -k 	+ ar_gnr*l.k 
	local deriv3  	 -l 	+ ar_gnr*l.l
	local deriv4  	 -kXk 	+ ar_gnr*l.kXk
	local deriv5  	 -lXl 	+ ar_gnr*l.lXl
	local deriv6  	 -mXm 	+ ar_gnr*l.mXm
	local deriv7  	 -kXl 	+ ar_gnr*l.kXl
	local deriv8  	 -kXm 	+ ar_gnr*l.kXm
	local deriv9  	 -lXm 	+ ar_gnr*l.lXm
	local deriv10  	-(l.y - _b[/bk]*l.k - _b[/bl]*l.l - _b[/bm]*l.m - _b[/bkk]*l.kXk - _b[/bll]*l.lXl - _b[/bmm]*l.mXm - _b[/bkl]*l.kXl - _b[/bkm]*l.kXm - _b[/blm]*l.lXm)
	
	tempfile pre
	save `pre'
	
	preserve
	gnr_wrap 1
	sum ar_gnr
	local ar = r(mean)
	restore
	
	gen ar_gnr = `ar'
	ar_dem
	
	
	
	

	tempvar tmp
	foreach var of varlist y k l m kXk lXl mXm kXl kXm lXm {
		egen `tmp' = mean(`var')
		replace `var' = `var'-`tmp'
		drop `tmp'
	}
	
	sort id t
	foreach d of numlist 1/9 {
		local count 0
		gen r`d' = `deriv`d''
	}
	
	gen mterm = ar_gnr*l.m
	
		
	//Reduced model
	
	cap reg r1 r2-r9, cluster(id)
	if _rc == 0 {
		local red_r2 	= e(r2)
		local red_rss	= e(rss)
	}
	else {
		local red_r2 .
		local red_rss .
	}
	
	//Full model
	cap reg r1 r2-r9 `instruments', cluster(id)
	if _rc == 0 {
		local full_r2 	= e(r2)
		local full_rss 	= e(rss)
		
		
		cap local partialr2 = (`red_rss'-`full_rss')/`red_rss'
		if _rc != 0 {
			local partialr2 .
		}
	}
	else {
		local full_r2 	.
		local full_rss 	.
	}
	
	cap test `instruments'
	if _rc == 0 {
		local teststat 	= r(F)
		local testp		= r(p)
	}
	else {
		local teststat 	.
		local testp		.
	}
	
	use `pre', clear
	
	gen `stub'_redr2 	= `red_r2'
	gen `stub'_fullr2 	= `full_r2'
	gen `stub'_pr2		= `partialr2'
	gen `stub'_F		= `teststat'
	gen `stub'_pval		= `testp'
end


/************************************************************************
Program: direct_id_test_2step
Purpose: Runs a two-step version of the advanced direct ID tests
************************************************************************/

cap program drop direct_id_test_2step
program direct_id_test_2step
	syntax varlist [if], Instruments(varlist) [stub(name)]
	
	//If they don't pass a stub, fill something in
	if mi("`stub'") {
		local stub idtest_2step
	}
	
	//Get intermediates and all the other variables (terms in production function)
	gettoken m rest: varlist
	
	preserve

		sort id t
		
		//Construct lags and also build up a statement
		local statement ""
		local count 0
		foreach var of varlist `rest' {
			gen LAG`var' = l.`var'
			
			//Build up NL statement
			local ++count
			local statement "`statement' + {a`count'}*( {rho}*LAG`var' - `var'   )"
		}
		
		gen LAG`m' = l.`m'

		
		
		//Drop anything with missing variables
		egen anymiss = rowmiss(`m' `rest' LAG* `instruments')
		drop if anymiss > 0
		
		
		
		//Demean
		tempvar tmp
		foreach var of varlist `m' `rest' LAG* `instruments' {
			egen `tmp' = mean(`var')
			replace `var' = `var'-`tmp'
			drop `tmp'
		}
		
		
		//Reduced model
		//nl (`m' = {cons} + {rho}*LAG`m' `statement' ), cluster(id)
		nl (`m' =  {rho}*LAG`m' `statement' ), cluster(id)
		predict resid, resid
		
		reg resid `instruments'
		local resr2 = e(r2)
		local resf 	= e(F)
		
	restore
	
	gen `stub'_resr2 	= `resr2'
	gen `stub'_resF 	= `resf'

end
